#!/bin/sh

#!/bin/sh

# Get article from seekalpha.com and convert it to tidy text format.
# Author : James Tan
# Copyright: "(C) 20012-2013 James Tan. GNU GPL 3."
# Version: v0.14 (beta)

# History (update half a month)
# v0.1  - 11/20/2012
#   basic functionalities
# v0.14 - 01/14/2013  
#  fix a bug to find start point of article

#set -x 
URL=$(echo $1 | tr '&' '\n' | tr '?' '\n' | grep '/article/')

wget -o wget.log -O page.html --keep-session-cookies --save-cookies wget.cookie "$URL"

sed 's|<Print>||' page.html > page.tidy.html
./tidy -f tidy.log -m page.tidy.html 
./html2text.py -b 0 -i 2 page.tidy.html > page.txt
# clean up the text file
sed 's/\\n//g' page.txt | sed '/^$/d' | sed '/^!.*$/d' > page.tmp.txt
# remove all lines before article
sl=$(grep -n "^#" page.tmp.txt | sed -n '2p' | cut -d ':' -f 1)
if [ -z "$sl" ]; then
  sl=$(grep -n "^#" page.tmp.txt | sed -n '1p' | cut -d ':' -f 1)
fi
sl=$((sl-1))
sed "1,${sl}d" page.tmp.txt > page.txt
# remove all lines after article
el=$(grep -n "^Tagged" page.txt | cut -d ':' -f 1)
sed -n "1,${el}p" page.txt > page.tmp.txt
# remove all the links and consolidate empty lines
#sed 's/.*http:\/\/.*//g' page.tmp.txt | sed '/^$/d;G' | sed '/./,/^$/!d' > page.txt
sed 's/[(]http:\/\/[^)]*//g' page.tmp.txt | sed '/^$/d;G' | sed '/./,/^$/!d' > page.txt
#sed 's/.*http:\/\/.*//g' page.tmp.txt > page.txt

# get comments
page_id=$(basename "$URL" | cut -d '-' -f 1)
comment_url="http://seekingalpha.com/memcached/get_first_comments/$page_id"

wget -o wget.log -O comment.html --load-cookies wget.cookie $comment_url
./tidy -f tidy.log -m comment.html
./html2text.py -b 0 -i 2 comment.html > comment.txt
sed 's/\\n//g' comment.txt | sed '/^$/d' | sed '/^!.*$/d' | sed '/^[ ]*$/d' | sed 's/\\t//g' | sed 's/.*http:\/\/.*//g'> comment.tmp.txt

cat comment.tmp.txt >> page.txt
rm page.html page.tidy.html page.tmp.txt comment.html comment.txt comment.tmp.txt

